<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">

<title>class YARP::LexCompat - yarp: Ruby Standard Library Documentation</title>


<script src="../js/navigation.js" defer></script>
<script src="../js/search.js" defer></script>
<script src="../js/search_index.js" defer></script>
<script src="../js/searcher.js" defer></script>
<script src="../js/darkfish.js" defer></script>

<script src="../js/jquery-3.2.0.min.js"></script>

<script src="../js/vue.min.js"></script>
<script src="../js/js.cookie.min.js"></script>

<link href="../css/fonts.css" rel="stylesheet">
<link id='rdoccss' href="../css/rdoc.css" rel="stylesheet">
<link href="../css/carbon17.css" rel="stylesheet">

<script type="text/javascript">
  var rdoc_rel_prefix = "../";
  var index_rel_prefix = "../";
  var darkModeCsseHref = "../css/rdoc-dm.css"
  var defaultModeCssHref = "../css/rdoc.css"
  // var cssDarkmode = Cookies.get('darkmode');
  
  if( Cookies.get("darkmode") == "true") {
	$('#rdoccss').attr("href", darkModeCsseHref);
}

//  https://cssdeck.com/blog/simple-jquery-stylesheet-switcher/

document.write('<style type="text/css">body{display:none}</style>');

</script>


</head>
<body id="top" role="document" class="class">
  <!-- this is class.html -->

  <div id='actionbar' >
    <div class='wrapper mdiv'>
      <ul class='grids g0'></ul>
    </div> 
    <!-- VERSION HEADER for 3.3.0.preview2 NOT FOUND -->
  </div> <!-- end action bar -->

  <div class='wrapper hdiv'>

    


    <nav id='vapp' role="navigation">
    <div id="project-navigation">
      <div id="home-section" role="region" title="Quick navigation" class="nav-section">
  <h2><a href="../index.html" rel="home">Home</a></h2>

  <div id="table-of-contents-navigation"  >
    <a href="../table_of_contents.html#pages">Pages</a>
    <a href="../table_of_contents.html#classes">Classes</a>
    <a href="../table_of_contents.html#methods">Methods</a>
  </div>
</div>

      <div id="search-section" role="search" class="project-section initially-hidden">
  <form action="#" method="get" accept-charset="utf-8">
    <div id="search-field-wrapper">
      <input id="search-field" role="combobox" aria-label="Search"
             aria-autocomplete="list" aria-controls="search-results"
             type="text" name="search" placeholder="Search" spellcheck="false"
             title="Type to search, Up and Down to navigate, Enter to load">
    </div>

    <ul id="search-results" aria-label="Search Results"
        aria-busy="false" aria-expanded="false"
        aria-atomic="false" class="initially-hidden"></ul>
  </form>
</div>

    </div>


    

    <button id='toggleThing' @click="toggleNav()" >Show/hide navigation</button>
    <div :class="isOpen ? 'block' : 'hidden' " id='toggleMe'>
      <div id="class-metadata">
        
        
<div id="parent-class-section" class="nav-section">
  <h3>Parent</h3>

  <p class="link"><a href="../Object.html">Object</a>
</div>

        
        
        
<!-- Method Quickref -->
<div id="method-list-section" class="nav-section">
  <h3>Methods</h3>

  <ul class="link-list" role="directory">
    <li ><a href="#method-c-new">::new</a>
    <li ><a href="#method-i-result">#result</a>
  </ul>
</div>

      </div>
     </div>
    </nav>


    <div id='extraz'><div class='adzbox-index'  >
      
     </div>         
    </div>

    <main role="main" aria-labelledby="class-YARP::LexCompat">
    <h1 id="class-YARP::LexCompat" class="class">
      class YARP::LexCompat
    </h1>

    <section class="description">
    
<p>This class is responsible for lexing the source using <a href="../YARP.html"><code>YARP</code></a> and then converting those tokens to be compatible with Ripper. In the vast majority of cases, this is a one-to-one mapping of the token type. Everything else generally lines up. However, there are a few cases that require special handling.</p>

    </section>

      <section id="5Buntitled-5D" class="documentation-section">


      <section class="constants-list">
      <header>
      <h3>Constants</h3>
      </header>
      <dl>
          <dt id="RIPPER">RIPPER
          <dd><p>This is a mapping of <a href="../YARP.html"><code>YARP</code></a> token types to Ripper token types. This is a many-to-one mapping because we split up our token types, whereas Ripper tends to group them.</p>
        </dl>
        </section>

        <section class="attribute-method-details" class="method-section">
        <header>
        <h3>Attributes</h3>
        </header>

          <div id="attribute-i-filepath" class="method-detail">
            <div class="method-heading attribute-method-heading">
              <span class="method-name">filepath</span><span
                class="attribute-access-type">[R]</span>
            </div>

            <div class="method-description">
              
              </div>
            </div>
          <div id="attribute-i-source" class="method-detail">
            <div class="method-heading attribute-method-heading">
              <span class="method-name">source</span><span
                class="attribute-access-type">[R]</span>
            </div>

            <div class="method-description">
              
              </div>
            </div>
            </section>


                <section id="public-class-5Buntitled-5D-method-details" class="method-section">
                <header>
                <h3>Public Class Methods</h3>
                </header>

                  <div id="method-c-new" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">new</span><span
                                class="method-args">(source, filepath = &quot;&quot;)</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              

                              <div class="method-source-code" id="new-source">
            <pre><span class="ruby-comment"># File yarp/lex_compat.rb, line 554</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">initialize</span>(<span class="ruby-identifier">source</span>, <span class="ruby-identifier">filepath</span> = <span class="ruby-string">&quot;&quot;</span>)
  <span class="ruby-ivar">@source</span> = <span class="ruby-identifier">source</span>
  <span class="ruby-ivar">@filepath</span> = <span class="ruby-identifier">filepath</span> <span class="ruby-operator">||</span> <span class="ruby-string">&quot;&quot;</span>
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                          </section>

                <section id="public-instance-5Buntitled-5D-method-details" class="method-section">
                <header>
                <h3>Public Instance Methods</h3>
                </header>

                  <div id="method-i-result" class="method-detail ">
                            <div class="method-heading">
                              <span class="method-name">result</span><span
                                class="method-args">()</span>
                              <span class="method-click-advice">click to toggle source</span>
                            </div>

                            <div class="method-description">
                              

                              <div class="method-source-code" id="result-source">
            <pre><span class="ruby-comment"># File yarp/lex_compat.rb, line 559</span>
<span class="ruby-keyword">def</span> <span class="ruby-identifier ruby-title">result</span>
  <span class="ruby-identifier">tokens</span> = []

  <span class="ruby-identifier">state</span> = <span class="ruby-value">:default</span>
  <span class="ruby-identifier">heredoc_stack</span> = [[]]

  <span class="ruby-identifier">result</span> = <span class="ruby-constant">YARP</span>.<span class="ruby-identifier">lex</span>(<span class="ruby-identifier">source</span>, <span class="ruby-ivar">@filepath</span>)
  <span class="ruby-identifier">result_value</span> = <span class="ruby-identifier">result</span>.<span class="ruby-identifier">value</span>
  <span class="ruby-identifier">previous_state</span> = <span class="ruby-keyword">nil</span>

  <span class="ruby-comment"># In previous versions of Ruby, Ripper wouldn&#39;t flush the bom before the</span>
  <span class="ruby-comment"># first token, so we had to have a hack in place to account for that. This</span>
  <span class="ruby-comment"># checks for that behavior.</span>
  <span class="ruby-identifier">bom_flushed</span> = <span class="ruby-constant">Ripper</span>.<span class="ruby-identifier">lex</span>(<span class="ruby-string">&quot;\xEF\xBB\xBF# test&quot;</span>)[<span class="ruby-value">0</span>][<span class="ruby-value">0</span>][<span class="ruby-value">1</span>] <span class="ruby-operator">==</span> <span class="ruby-value">0</span>
  <span class="ruby-identifier">bom</span> = <span class="ruby-identifier">source</span>.<span class="ruby-identifier">byteslice</span>(<span class="ruby-value">0</span><span class="ruby-operator">..</span><span class="ruby-value">2</span>) <span class="ruby-operator">==</span> <span class="ruby-string">&quot;\xEF\xBB\xBF&quot;</span>

  <span class="ruby-identifier">result_value</span>.<span class="ruby-identifier">each_with_index</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span>(<span class="ruby-identifier">token</span>, <span class="ruby-identifier">lex_state</span>), <span class="ruby-identifier">index</span><span class="ruby-operator">|</span>
    <span class="ruby-identifier">lineno</span> = <span class="ruby-identifier">token</span>.<span class="ruby-identifier">location</span>.<span class="ruby-identifier">start_line</span>
    <span class="ruby-identifier">column</span> = <span class="ruby-identifier">token</span>.<span class="ruby-identifier">location</span>.<span class="ruby-identifier">start_column</span>

    <span class="ruby-comment"># If there&#39;s a UTF-8 byte-order mark as the start of the file, then for</span>
    <span class="ruby-comment"># certain tokens ripper sets the first token back by 3 bytes. It also</span>
    <span class="ruby-comment"># keeps the byte order mark in the first token&#39;s value. This is weird,</span>
    <span class="ruby-comment"># and I don&#39;t want to mirror that in our parser. So instead, we&#39;ll match</span>
    <span class="ruby-comment"># up the columns and values here.</span>
    <span class="ruby-keyword">if</span> <span class="ruby-identifier">bom</span> <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-identifier">lineno</span> <span class="ruby-operator">==</span> <span class="ruby-value">1</span>
      <span class="ruby-identifier">column</span> <span class="ruby-operator">-=</span> <span class="ruby-value">3</span>

      <span class="ruby-keyword">if</span> <span class="ruby-identifier">index</span> <span class="ruby-operator">==</span> <span class="ruby-value">0</span> <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-identifier">column</span> <span class="ruby-operator">==</span> <span class="ruby-value">0</span> <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-operator">!</span><span class="ruby-identifier">bom_flushed</span>
        <span class="ruby-identifier">flushed</span> =
          <span class="ruby-keyword">case</span> <span class="ruby-identifier">token</span>.<span class="ruby-identifier">type</span>
          <span class="ruby-keyword">when</span> <span class="ruby-value">:BACK_REFERENCE</span>, <span class="ruby-value">:INSTANCE_VARIABLE</span>, <span class="ruby-value">:CLASS_VARIABLE</span>,
              <span class="ruby-value">:GLOBAL_VARIABLE</span>, <span class="ruby-value">:NUMBERED_REFERENCE</span>, <span class="ruby-value">:PERCENT_LOWER_I</span>,
              <span class="ruby-value">:PERCENT_LOWER_X</span>, <span class="ruby-value">:PERCENT_LOWER_W</span>, <span class="ruby-value">:PERCENT_UPPER_I</span>,
              <span class="ruby-value">:PERCENT_UPPER_W</span>, <span class="ruby-value">:STRING_BEGIN</span>
            <span class="ruby-keyword">true</span>
          <span class="ruby-keyword">when</span> <span class="ruby-value">:REGEXP_BEGIN</span>, <span class="ruby-value">:SYMBOL_BEGIN</span>
            <span class="ruby-identifier">token</span>.<span class="ruby-identifier">value</span>.<span class="ruby-identifier">start_with?</span>(<span class="ruby-string">&quot;%&quot;</span>)
          <span class="ruby-keyword">else</span>
            <span class="ruby-keyword">false</span>
          <span class="ruby-keyword">end</span>

        <span class="ruby-keyword">unless</span> <span class="ruby-identifier">flushed</span>
          <span class="ruby-identifier">column</span> <span class="ruby-operator">-=</span> <span class="ruby-value">3</span>
          <span class="ruby-identifier">value</span> = <span class="ruby-identifier">token</span>.<span class="ruby-identifier">value</span>
          <span class="ruby-identifier">value</span>.<span class="ruby-identifier">prepend</span>(<span class="ruby-constant">String</span>.<span class="ruby-identifier">new</span>(<span class="ruby-string">&quot;\xEF\xBB\xBF&quot;</span>, <span class="ruby-value">encoding:</span> <span class="ruby-identifier">value</span>.<span class="ruby-identifier">encoding</span>))
        <span class="ruby-keyword">end</span>
      <span class="ruby-keyword">end</span>
    <span class="ruby-keyword">end</span>

    <span class="ruby-identifier">event</span> = <span class="ruby-constant">RIPPER</span>.<span class="ruby-identifier">fetch</span>(<span class="ruby-identifier">token</span>.<span class="ruby-identifier">type</span>)
    <span class="ruby-identifier">value</span> = <span class="ruby-identifier">token</span>.<span class="ruby-identifier">value</span>
    <span class="ruby-identifier">lex_state</span> = <span class="ruby-constant">Ripper</span><span class="ruby-operator">::</span><span class="ruby-constant">Lexer</span><span class="ruby-operator">::</span><span class="ruby-constant">State</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">lex_state</span>)

    <span class="ruby-identifier">token</span> =
      <span class="ruby-keyword">case</span> <span class="ruby-identifier">event</span>
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on___end__</span>
        <span class="ruby-constant">EndContentToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_comment</span>
        <span class="ruby-constant">IgnoreStateToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_heredoc_end</span>
        <span class="ruby-comment"># Heredoc end tokens can be emitted in an odd order, so we don&#39;t</span>
        <span class="ruby-comment"># want to bother comparing the state on them.</span>
        <span class="ruby-constant">IgnoreStateToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_ident</span>
        <span class="ruby-keyword">if</span> <span class="ruby-identifier">lex_state</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Ripper</span><span class="ruby-operator">::</span><span class="ruby-constant">EXPR_END</span>
          <span class="ruby-comment"># If we have an identifier that follows a method name like:</span>
          <span class="ruby-comment">#</span>
          <span class="ruby-comment">#     def foo bar</span>
          <span class="ruby-comment">#</span>
          <span class="ruby-comment"># then Ripper will mark bar as END|LABEL if there is a local in a</span>
          <span class="ruby-comment"># parent scope named bar because it hasn&#39;t pushed the local table</span>
          <span class="ruby-comment"># yet. We do this more accurately, so we need to allow comparing</span>
          <span class="ruby-comment"># against both END and END|LABEL.</span>
          <span class="ruby-constant">ParamToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
        <span class="ruby-keyword">elsif</span> <span class="ruby-identifier">lex_state</span> <span class="ruby-operator">==</span> <span class="ruby-constant">Ripper</span><span class="ruby-operator">::</span><span class="ruby-constant">EXPR_END</span> <span class="ruby-operator">|</span> <span class="ruby-constant">Ripper</span><span class="ruby-operator">::</span><span class="ruby-constant">EXPR_LABEL</span>
          <span class="ruby-comment"># In the event that we&#39;re comparing identifiers, we&#39;re going to</span>
          <span class="ruby-comment"># allow a little divergence. Ripper doesn&#39;t account for local</span>
          <span class="ruby-comment"># variables introduced through named captures in regexes, and we</span>
          <span class="ruby-comment"># do, which accounts for this difference.</span>
          <span class="ruby-constant">IdentToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
        <span class="ruby-keyword">else</span>
          <span class="ruby-constant">Token</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
        <span class="ruby-keyword">end</span>
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_embexpr_end</span>
        <span class="ruby-constant">IgnoreStateToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_ignored_nl</span>
        <span class="ruby-comment"># Ignored newlines can occasionally have a LABEL state attached to</span>
        <span class="ruby-comment"># them which doesn&#39;t actually impact anything. We don&#39;t mirror that</span>
        <span class="ruby-comment"># state so we ignored it.</span>
        <span class="ruby-constant">IgnoredNewlineToken</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_regexp_end</span>
        <span class="ruby-comment"># On regex end, Ripper scans and then sets end state, so the ripper</span>
        <span class="ruby-comment"># lexed output is begin, when it should be end. YARP sets lex state</span>
        <span class="ruby-comment"># correctly to end state, but we want to be able to compare against</span>
        <span class="ruby-comment"># Ripper&#39;s lexed state. So here, if it&#39;s a regexp end token, we</span>
        <span class="ruby-comment"># output the state as the previous state, solely for the sake of</span>
        <span class="ruby-comment"># comparison.</span>
        <span class="ruby-identifier">previous_token</span> = <span class="ruby-identifier">result_value</span>[<span class="ruby-identifier">index</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>][<span class="ruby-value">0</span>]
        <span class="ruby-identifier">lex_state</span> =
          <span class="ruby-keyword">if</span> <span class="ruby-constant">RIPPER</span>.<span class="ruby-identifier">fetch</span>(<span class="ruby-identifier">previous_token</span>.<span class="ruby-identifier">type</span>) <span class="ruby-operator">==</span> <span class="ruby-value">:on_embexpr_end</span>
            <span class="ruby-comment"># If the previous token is embexpr_end, then we have to do even</span>
            <span class="ruby-comment"># more processing. The end of an embedded expression sets the</span>
            <span class="ruby-comment"># state to the state that it had at the beginning of the</span>
            <span class="ruby-comment"># embedded expression. So we have to go and find that state and</span>
            <span class="ruby-comment"># set it here.</span>
            <span class="ruby-identifier">counter</span> = <span class="ruby-value">1</span>
            <span class="ruby-identifier">current_index</span> = <span class="ruby-identifier">index</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>

            <span class="ruby-keyword">until</span> <span class="ruby-identifier">counter</span> <span class="ruby-operator">==</span> <span class="ruby-value">0</span>
              <span class="ruby-identifier">current_index</span> <span class="ruby-operator">-=</span> <span class="ruby-value">1</span>
              <span class="ruby-identifier">current_event</span> = <span class="ruby-constant">RIPPER</span>.<span class="ruby-identifier">fetch</span>(<span class="ruby-identifier">result_value</span>[<span class="ruby-identifier">current_index</span>][<span class="ruby-value">0</span>].<span class="ruby-identifier">type</span>)
              <span class="ruby-identifier">counter</span> <span class="ruby-operator">+=</span> { <span class="ruby-value">on_embexpr_beg:</span> <span class="ruby-value">-1</span>, <span class="ruby-value">on_embexpr_end:</span> <span class="ruby-value">1</span> }[<span class="ruby-identifier">current_event</span>] <span class="ruby-operator">||</span> <span class="ruby-value">0</span>
            <span class="ruby-keyword">end</span>

            <span class="ruby-constant">Ripper</span><span class="ruby-operator">::</span><span class="ruby-constant">Lexer</span><span class="ruby-operator">::</span><span class="ruby-constant">State</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">result_value</span>[<span class="ruby-identifier">current_index</span>][<span class="ruby-value">1</span>])
          <span class="ruby-keyword">else</span>
            <span class="ruby-identifier">previous_state</span>
          <span class="ruby-keyword">end</span>

        <span class="ruby-constant">Token</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_eof</span>
        <span class="ruby-identifier">previous_token</span> = <span class="ruby-identifier">result_value</span>[<span class="ruby-identifier">index</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span>][<span class="ruby-value">0</span>]

        <span class="ruby-comment"># If we&#39;re at the end of the file and the previous token was a</span>
        <span class="ruby-comment"># comment and there is still whitespace after the comment, then</span>
        <span class="ruby-comment"># Ripper will append a on_nl token (even though there isn&#39;t</span>
        <span class="ruby-comment"># necessarily a newline). We mirror that here.</span>
        <span class="ruby-identifier">start_offset</span> = <span class="ruby-identifier">previous_token</span>.<span class="ruby-identifier">location</span>.<span class="ruby-identifier">end_offset</span>
        <span class="ruby-identifier">end_offset</span> = <span class="ruby-identifier">token</span>.<span class="ruby-identifier">location</span>.<span class="ruby-identifier">start_offset</span>

        <span class="ruby-keyword">if</span> <span class="ruby-identifier">previous_token</span>.<span class="ruby-identifier">type</span> <span class="ruby-operator">==</span> <span class="ruby-value">:COMMENT</span> <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-identifier">start_offset</span> <span class="ruby-operator">&lt;</span> <span class="ruby-identifier">end_offset</span>
          <span class="ruby-keyword">if</span> <span class="ruby-identifier">bom</span>
            <span class="ruby-identifier">start_offset</span> <span class="ruby-operator">+=</span> <span class="ruby-value">3</span>
            <span class="ruby-identifier">end_offset</span> <span class="ruby-operator">+=</span> <span class="ruby-value">3</span>
          <span class="ruby-keyword">end</span>

          <span class="ruby-identifier">tokens</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-constant">Token</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-value">0</span>], <span class="ruby-value">:on_nl</span>, <span class="ruby-identifier">source</span>.<span class="ruby-identifier">byteslice</span>(<span class="ruby-identifier">start_offset</span><span class="ruby-operator">...</span><span class="ruby-identifier">end_offset</span>), <span class="ruby-identifier">lex_state</span>])
        <span class="ruby-keyword">end</span>

        <span class="ruby-constant">Token</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">else</span>
        <span class="ruby-constant">Token</span>.<span class="ruby-identifier">new</span>([[<span class="ruby-identifier">lineno</span>, <span class="ruby-identifier">column</span>], <span class="ruby-identifier">event</span>, <span class="ruby-identifier">value</span>, <span class="ruby-identifier">lex_state</span>])
      <span class="ruby-keyword">end</span>

    <span class="ruby-identifier">previous_state</span> = <span class="ruby-identifier">lex_state</span>

    <span class="ruby-comment"># The order in which tokens appear in our lexer is different from the</span>
    <span class="ruby-comment"># order that they appear in Ripper. When we hit the declaration of a</span>
    <span class="ruby-comment"># heredoc in YARP, we skip forward and lex the rest of the content of</span>
    <span class="ruby-comment"># the heredoc before going back and lexing at the end of the heredoc</span>
    <span class="ruby-comment"># identifier.</span>
    <span class="ruby-comment">#</span>
    <span class="ruby-comment"># To match up to ripper, we keep a small state variable around here to</span>
    <span class="ruby-comment"># track whether we&#39;re in the middle of a heredoc or not. In this way we</span>
    <span class="ruby-comment"># can shuffle around the token to match Ripper&#39;s output.</span>
    <span class="ruby-keyword">case</span> <span class="ruby-identifier">state</span>
    <span class="ruby-keyword">when</span> <span class="ruby-value">:default</span>
      <span class="ruby-comment"># The default state is when there are no heredocs at all. In this</span>
      <span class="ruby-comment"># state we can append the token to the list of tokens and move on.</span>
      <span class="ruby-identifier">tokens</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>

      <span class="ruby-comment"># If we get the declaration of a heredoc, then we open a new heredoc</span>
      <span class="ruby-comment"># and move into the heredoc_opened state.</span>
      <span class="ruby-keyword">if</span> <span class="ruby-identifier">event</span> <span class="ruby-operator">==</span> <span class="ruby-value">:on_heredoc_beg</span>
        <span class="ruby-identifier">state</span> = <span class="ruby-value">:heredoc_opened</span>
        <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-constant">Heredoc</span>.<span class="ruby-identifier">build</span>(<span class="ruby-identifier">token</span>)
      <span class="ruby-keyword">end</span>
    <span class="ruby-keyword">when</span> <span class="ruby-value">:heredoc_opened</span>
      <span class="ruby-comment"># The heredoc_opened state is when we&#39;ve seen the declaration of a</span>
      <span class="ruby-comment"># heredoc and are now lexing the body of the heredoc. In this state we</span>
      <span class="ruby-comment"># push tokens onto the most recently created heredoc.</span>
      <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span>.<span class="ruby-identifier">last</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>

      <span class="ruby-keyword">case</span> <span class="ruby-identifier">event</span>
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_heredoc_beg</span>
        <span class="ruby-comment"># If we receive a heredoc declaration while lexing the body of a</span>
        <span class="ruby-comment"># heredoc, this means we have nested heredocs. In this case we&#39;ll</span>
        <span class="ruby-comment"># push a new heredoc onto the stack and stay in the heredoc_opened</span>
        <span class="ruby-comment"># state since we&#39;re now lexing the body of the new heredoc.</span>
        <span class="ruby-identifier">heredoc_stack</span> <span class="ruby-operator">&lt;&lt;</span> [<span class="ruby-constant">Heredoc</span>.<span class="ruby-identifier">build</span>(<span class="ruby-identifier">token</span>)]
      <span class="ruby-keyword">when</span> <span class="ruby-value">:on_heredoc_end</span>
        <span class="ruby-comment"># If we receive the end of a heredoc, then we&#39;re done lexing the</span>
        <span class="ruby-comment"># body of the heredoc. In this case we now have a completed heredoc</span>
        <span class="ruby-comment"># but need to wait for the next newline to push it into the token</span>
        <span class="ruby-comment"># stream.</span>
        <span class="ruby-identifier">state</span> = <span class="ruby-value">:heredoc_closed</span>
      <span class="ruby-keyword">end</span>
    <span class="ruby-keyword">when</span> <span class="ruby-value">:heredoc_closed</span>
      <span class="ruby-keyword">if</span> <span class="ruby-node">%i[on_nl on_ignored_nl on_comment]</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">event</span>) <span class="ruby-operator">||</span> (<span class="ruby-identifier">event</span> <span class="ruby-operator">==</span> <span class="ruby-value">:on_tstring_content</span> <span class="ruby-operator">&amp;&amp;</span> <span class="ruby-identifier">value</span>.<span class="ruby-identifier">end_with?</span>(<span class="ruby-string">&quot;\n&quot;</span>))
        <span class="ruby-keyword">if</span> <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">size</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">1</span>
          <span class="ruby-identifier">flushing</span> = <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">pop</span>
          <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span>.<span class="ruby-identifier">last</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>

          <span class="ruby-identifier">flushing</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">heredoc</span><span class="ruby-operator">|</span>
            <span class="ruby-identifier">heredoc</span>.<span class="ruby-identifier">to_a</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">flushed_token</span><span class="ruby-operator">|</span>
              <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span>.<span class="ruby-identifier">last</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">flushed_token</span>
            <span class="ruby-keyword">end</span>
          <span class="ruby-keyword">end</span>

          <span class="ruby-identifier">state</span> = <span class="ruby-value">:heredoc_opened</span>
          <span class="ruby-keyword">next</span>
        <span class="ruby-keyword">end</span>
      <span class="ruby-keyword">elsif</span> <span class="ruby-identifier">event</span> <span class="ruby-operator">==</span> <span class="ruby-value">:on_heredoc_beg</span>
        <span class="ruby-identifier">tokens</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>
        <span class="ruby-identifier">state</span> = <span class="ruby-value">:heredoc_opened</span>
        <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-constant">Heredoc</span>.<span class="ruby-identifier">build</span>(<span class="ruby-identifier">token</span>)
        <span class="ruby-keyword">next</span>
      <span class="ruby-keyword">elsif</span> <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">size</span> <span class="ruby-operator">&gt;</span> <span class="ruby-value">1</span>
        <span class="ruby-identifier">heredoc_stack</span>[<span class="ruby-value">-2</span>].<span class="ruby-identifier">last</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>
        <span class="ruby-keyword">next</span>
      <span class="ruby-keyword">end</span>

      <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">heredoc</span><span class="ruby-operator">|</span>
        <span class="ruby-identifier">tokens</span>.<span class="ruby-identifier">concat</span>(<span class="ruby-identifier">heredoc</span>.<span class="ruby-identifier">to_a</span>)
      <span class="ruby-keyword">end</span>

      <span class="ruby-identifier">heredoc_stack</span>.<span class="ruby-identifier">last</span>.<span class="ruby-identifier">clear</span>
      <span class="ruby-identifier">state</span> = <span class="ruby-value">:default</span>

      <span class="ruby-identifier">tokens</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">token</span>
    <span class="ruby-keyword">end</span>
  <span class="ruby-keyword">end</span>

  <span class="ruby-comment"># Drop the EOF token from the list</span>
  <span class="ruby-identifier">tokens</span> = <span class="ruby-identifier">tokens</span>[<span class="ruby-value">0</span><span class="ruby-operator">...</span><span class="ruby-value">-1</span>]

  <span class="ruby-comment"># We sort by location to compare against Ripper&#39;s output</span>
  <span class="ruby-identifier">tokens</span>.<span class="ruby-identifier">sort_by!</span>(<span class="ruby-operator">&amp;</span><span class="ruby-value">:location</span>)

  <span class="ruby-keyword">if</span> <span class="ruby-identifier">result_value</span>.<span class="ruby-identifier">size</span> <span class="ruby-operator">-</span> <span class="ruby-value">1</span> <span class="ruby-operator">&gt;</span> <span class="ruby-identifier">tokens</span>.<span class="ruby-identifier">size</span>
    <span class="ruby-identifier">raise</span> <span class="ruby-constant">StandardError</span>, <span class="ruby-string">&quot;Lost tokens when performing lex_compat&quot;</span>
  <span class="ruby-keyword">end</span>

  <span class="ruby-constant">ParseResult</span>.<span class="ruby-identifier">new</span>(<span class="ruby-identifier">tokens</span>, <span class="ruby-identifier">result</span>.<span class="ruby-identifier">comments</span>, <span class="ruby-identifier">result</span>.<span class="ruby-identifier">errors</span>, <span class="ruby-identifier">result</span>.<span class="ruby-identifier">warnings</span>, [])
<span class="ruby-keyword">end</span></pre>
                              </div>
                            </div>


                          </div>

                          </section>

              </section>
              </main>



            </div>  <!--  class='wrapper hdiv' -->


<footer id="validator-badges" role="contentinfo">
<p><a href="https://validator.w3.org/check/referer">Validate</a></p>
<p>Generated by <a href="https://ruby.github.io/rdoc/">RDoc</a> 6.4.0.</p>
<p>Based on <a href="https://github.com/ged/darkfish/">Darkfish</a> by <a href="http://deveiate.org">Michael Granger</a>.</p>

  
    <p><p><a href="https://ruby-doc.org">Ruby-doc.org</a> is a service of <a href="https://jamesbritt.com">James Britt</a> and <a href="https://neurogami.com">Neurogami</a>, purveyors of fine <a href='https://jamesbritt.bandcamp.com/'>dance noise</a></p>
</p>
  
  </footer>

<script type="text/javascript">


  let ads  = $("#carbonads-container").children().detach();


  function swapMode() {
    var cookieName = 'darkmode';
    var cssDarkmode = Cookies.get(cookieName);
    console.log("***** swapMode! " + cssDarkmode + " *****");


    if (cssDarkmode == "true") {
      console.log("We have dark mode, set the css to light ...");
      $('#rdoccss').attr("href", defaultModeCssHref);
      $('#cssSelect').text("Dark mode");
      cssDarkmode = "false";
      console.log("swapMode! Now set cookie to " + cssDarkmode);
      Cookies.set(cookieName, cssDarkmode);

    } else {
      console.log("We not have dark mode, set the css to dark ...");
      $('#rdoccss').attr("href", darkModeCsseHref);
      $('#cssSelect').text("Light mode");
      cssDarkmode = "true";
      console.log("swapMode! Now set cookie to " + cssDarkmode);
      Cookies.set(cookieName, cssDarkmode);

    }

    console.log("  --------------- ");
  }


const vueCssApp = new Vue({
el: '#menubar',
data: {
isDark: false
},
methods: {
toggleClass: function(event){
this.isDark = !this.isDark;
}
}
})

const vueApp = new Vue({
el: '#vapp',
data: { 
isOpen: true
},

mounted() {
this.handleResize();
this.manage_mob_classes();
window.addEventListener('resize', this.handleResize)
//this.isOpen !=  (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent));
},
destroyed() {
window.removeEventListener('resize', this.handleResize)
},
created() {
//manage_mob_classes();
},

methods : {
isMobile() {
  return (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent));
},

  handleResize() {
    if (!this.isMobile()) {
      this.isOpen = window.innerWidth > 800;
    }
  },

  manage_mob_classes() {
    if (/Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)) {
      $("nav").addClass("mob_nav");
      $("main").addClass("mob_main");
      $("#extraz").addClass("mob_extraz");
      $("#carbonads-container").addClass("mob_carbonads-container");
      this.isOpen  = false;
    } else {
      $("nav").removeClass("mob_nav") 
        $("main").removeClass("mob_main");
      $("#extraz").removeClass("mob_extraz");
      $("#carbonads-container").removeClass("mob_carbonads-container");
      this.isOpen  = true;
    }
  },

  toggleNav() {
    this.isOpen =! this.isOpen ;
    // alert("Toggle nav!");
    console.log("toggleNav() click: " + this.isOpen );
  }
}
})

$("#carbonads-container").append(ads);


$(function() {

    var darkmode = Cookies.get("darkmode");
    console.log("Document ready: " + darkmode);

    if ( darkmode  == "true" ) {
      $('#cssSelect').text("Light mode");
    } else {
      $('#cssSelect').text("Dark mode");
     }

    $('body').css('display','block');
    });

</script>

    
  </body> 
</html>

